﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.Serialization.Formatters.Binary;
using System.Linq;
using System.Text;
using System.IO;

namespace StockLearner
{
    class SimpleRLAgent : Agent
    {
        Hashtable m_policyHashTable;
        String m_policyHashFileName = "policySimpleRL.dat";

        double m_alpha = .001;
        double m_gamma = .01;
        double m_epsilon = 0.05;

        double m_rewardIntegrator = 0.0;

        public SimpleRLAgent():base()
        {
            if (System.IO.File.Exists(m_policyHashFileName))
            {
                BinaryFormatter l_formatter = new BinaryFormatter();
                FileStream l_file = new FileStream(m_policyHashFileName, FileMode.Open, FileAccess.ReadWrite);
                StreamReader l_reader = new StreamReader(l_file);
                m_policyHashTable = (Hashtable)l_formatter.Deserialize(l_reader.BaseStream);
            }
            else
            {
                m_policyHashTable = new Hashtable();
                List<Double> l_startingEntry = new List<Double>();

                for (int i = 0; i < 30; i++)
                {
                    l_startingEntry.Add(1.0);
                }

                String l_key = Utils.DblArrayToString(l_startingEntry.ToArray());
                Hashtable l_toAdd = new Hashtable();

                Random l_generator = new Random();
                double l_startQ = l_generator.NextDouble() * -1.0 ;
                for(double l_action = 5; l_action <= 15; l_action += .01)
                {
                    double l_actionToAdd = Math.Round(l_action / 10.0, 3);
                    l_toAdd.Add(l_actionToAdd, l_startQ);
                    l_startQ = l_generator.NextDouble() * -1.0;
                }

                m_policyHashTable.Add(l_key, l_toAdd);

                BinaryFormatter l_formatter = new BinaryFormatter();

                FileStream l_file = new FileStream(m_policyHashFileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
                StreamWriter l_writer = new StreamWriter(l_file);

                l_formatter.Serialize(l_writer.BaseStream, m_policyHashTable);
            }
        }

        public override List<YahooDataPoint> act(List<YahooDataPoint> p_state)
        {
            List<YahooDataPoint> l_toReturn = new List<YahooDataPoint>();

            double[] l_stateDblArr = new double[p_state.Count];

            for (int i = 0; i < p_state.Count; i++)
            {
                l_stateDblArr[i] = p_state[i].Close;
            }

            double l_predDeriv = 0.0;
            double l_avg = 0.0;

            for (int i = 0; i < l_stateDblArr.Length; i++)
            {
                l_avg = ((l_avg * i) + (l_stateDblArr[i])) / (i + 1);
            }

            for (int i = 1; i < l_stateDblArr.Length; i++)
            {
                l_predDeriv = ((l_predDeriv * i) + (l_stateDblArr[i] - l_stateDblArr[i - 1])) / (i + 1);
            }

            double l_prediction = (l_predDeriv * 30) + l_avg;
            l_prediction = Math.Round(l_prediction, 3);

            String l_key = Utils.DblArrayToString(l_stateDblArr);
            Hashtable l_stateHash;
            if (m_policyHashTable.ContainsKey(l_key))
            {
                l_stateHash = (Hashtable)m_policyHashTable[l_key];
                double l_greedyA = 1.0;
                double l_greedyA_Q = double.MinValue;

                foreach (DictionaryEntry l_entry in l_stateHash)
                {
                    double l_q = (double)l_entry.Value;

                    if (l_q > l_greedyA_Q)
                    {
                        l_greedyA = (double)l_entry.Key;
                        l_greedyA_Q = l_q;
                    }
                }

                l_prediction = Math.Round(l_greedyA, 3);
            }
            else
            {

                Hashtable l_hashToAdd = new Hashtable();
                Random l_generator = new Random();
                double l_startQ = l_generator.NextDouble() * -1.0;

                for (double l_action = 5; l_action <= 15; l_action += .01)
                {
                    double l_actionToAdd = Math.Round(l_action / 10.0, 3);
                    l_hashToAdd.Add(l_actionToAdd, l_startQ);
                    l_startQ = l_generator.NextDouble() * -1.0;
                }

                m_policyHashTable.Add(l_key, l_hashToAdd);
            }

            Random l_randGen = new Random();
            //if less than episilon, generate random around current predicition
            //otherwise, take greedy action
            if (l_randGen.NextDouble() < m_epsilon)
            {
                double l_randPred = Utils.GaussianRandomApprox(l_prediction, .001);
                if (l_randPred > 1.5)
                {
                    l_randPred = 1.5;
                }
                else if (l_randPred < .5)
                {
                    l_randPred = .5;
                }
                l_prediction = Math.Round(l_randPred, 3);
            }

            YahooDataPoint l_initState = p_state[p_state.Count - 1];
            YahooDataPoint l_toAdd = new YahooDataPoint();

            l_toAdd.High = l_initState.High;
            l_toAdd.Low = l_initState.Low;
            l_toAdd.Open = l_initState.Open;
            l_toAdd.Volume = l_initState.Volume;
            l_toAdd.Close = Math.Round(l_prediction, 3);
            l_toAdd.Timestamp = Utils.AddMinutesUnixTimeStamp(l_initState.Timestamp, 30.0);

            l_toReturn.Add(l_toAdd);

            return l_toReturn;
        }

        public override void saveKnowledge()
        {
            BinaryFormatter l_formatter = new BinaryFormatter();

            FileStream l_file = new FileStream(m_policyHashFileName, FileMode.OpenOrCreate, FileAccess.ReadWrite);
            StreamWriter l_writer = new StreamWriter(l_file);

            l_formatter.Serialize(l_writer.BaseStream, m_policyHashTable);
        }

        public override void processReward(List<YahooDataPoint> p_pred_a, List<YahooDataPoint> p_s, List<YahooDataPoint> p_sPrime)
        {
            double l_predictedVal = p_pred_a[0].Close;
            double l_normalizationVal = p_s[0].Close;
            double l_actualResult = p_sPrime[0].Close;
            double l_normalizedResult = l_actualResult / l_normalizationVal;

            List<YahooDataPoint> l_s_normed = Form1.GeneralizeAndDiscretize(p_s);
            List<YahooDataPoint> l_sPrime_normed = Form1.GeneralizeAndDiscretize(p_sPrime);

            l_normalizedResult = Math.Round(l_normalizedResult, 3);

            double l_reward = -1.0 * Math.Round(Math.Abs(l_normalizedResult - l_predictedVal), 3);

            m_rewardIntegrator += l_reward;

            List<YahooDataPoint> l_aPrimeList = act(l_sPrime_normed);
            double l_aPrime = Math.Round((l_aPrimeList[0].Close), 3);

            double l_q;
            double l_a = p_pred_a[0].Close;

            double[] l_stateDblArr = new double[l_s_normed.Count];

            for (int i = 0; i < l_s_normed.Count; i++)
            {
                l_stateDblArr[i] = Math.Round(l_s_normed[i].Close, 3);
            }

            double[] l_statePrimeDblArr = new double[l_sPrime_normed.Count];

            for (int i = 0; i < l_sPrime_normed.Count; i++)
            {
                l_statePrimeDblArr[i] = Math.Round(l_sPrime_normed[i].Close, 3);
            }

            String l_key = Utils.DblArrayToString(l_stateDblArr);
            String l_keyPrime = Utils.DblArrayToString(l_statePrimeDblArr);

            Hashtable l_stateHash = new Hashtable();
            Hashtable l_statePrimeHash = new Hashtable();

            if (m_policyHashTable.ContainsKey(l_key))
            {
                l_stateHash = (Hashtable)m_policyHashTable[l_key];
            }
            else
            {
            }

            if (m_policyHashTable.ContainsKey(l_keyPrime))
            {
                l_statePrimeHash = (Hashtable)m_policyHashTable[l_keyPrime];
            }
            else
            {
                Hashtable l_hashToAdd = new Hashtable();
                Random l_generator = new Random();
                double l_startQ = l_generator.NextDouble() * -1.0;

                for (double l_action = 5; l_action <= 15; l_action += .01)
                {
                    double l_actionToAdd = Math.Round(l_action / 10.0, 3);
                    l_hashToAdd.Add(l_actionToAdd, l_startQ);
                    l_startQ = l_generator.NextDouble() * -1.0;
                }

                m_policyHashTable.Add(l_keyPrime, l_hashToAdd);

                l_statePrimeHash = (Hashtable)m_policyHashTable[l_keyPrime];
            }

            l_q = (double)l_stateHash[l_a];
            double l_qPrime = (double)l_statePrimeHash[l_aPrime];
            l_q = l_q + (m_alpha * (l_reward + (m_gamma * l_qPrime) - l_q));

            l_stateHash[l_a] = l_q;

            m_policyHashTable[l_key] = l_stateHash;
        }

        public double getRewardSum()
        {
            return m_rewardIntegrator;
        }

        public void resetRewardSum()
        {
            m_rewardIntegrator = 0.0;
        }
    }
}
